const puppeteer = require('puppeteer');
const chalk = require('chalk');

const API = 'mtop.relationrecommend.aliexpressrecommend.recommend';
const fs = require('fs');
let result = [];
const log = console.log;
const keyword = process.argv.slice(2).join(' ') || 'thermos bottle';
const pageUrl = 'https://www.aliexpress.com';
const hasCache = {};
const maxTurnCount = 10;
let last = 1;
const RATE = 0;
const history = [];
const rootWords = [];
const mod = 'simple';
const needJson = false;
const map = {};

// 数组去重
function unique(arr) {
  return Array.from(new Set(arr));
}

const timestamp = new Date().getTime();

async function main() {
  const browser = await puppeteer.launch({ headless: true, args: [`--window-size=${1440},${1000}`] });
  const page = await browser.newPage();
  try {
    await page.goto(pageUrl, { waitUntil: 'networkidle2' });

    async function getTexts(arg) {
      await page.waitForSelector('.search-key-box input');
      await page.$eval('.search-key-box input', el => el.value = '');
      await page.type('.search-key-box input', arg, { delay: 30 });
      await page.waitForResponse(response => response.status() === 200 && response.url().includes(API))
      const texts = await page.evaluate(async () => {
        return [].slice.call(document.body.querySelectorAll('.ui-autocomplete li')).map(i => i.innerText).filter(i=>i);
      });
      result = unique(result.concat(texts));
      log(chalk.greenBright(arg), chalk.greenBright('完成'));
      hasCache[arg] = true;
      return texts;
    }

    // 输入一个词，开始遍历 26 个字母
    async function loopOneWord(w) {
      if (hasCache[w]) {
        log(w, 'skip');
        return;
      }

      let texts = await getTexts(w);
      if (needResearch()) {
        rootWords.push(w);
        const words = (function() {
          let i = 97, max = 122;
          const arr = [];
          while (i <= max) {
            arr.push(w + ' ' + String.fromCharCode(i++));
          }
          i = 0;
          while(i < 10) {
            arr.push(w + ' ' + i);
            i++;
          }
          return arr;
        })();
        async function loop() {
          texts = texts.concat(await getTexts(words.shift()));
          if (words.length) {
            await loop();
          }
        }
        await loop();
      }
      if (needJson) {
        let lis
        if (map[w]) {
          lis = unique(texts.concat(map[w]));
        } else {
          lis = unique(texts);
        }
        map[w] = lis.filter(i => i !== w).filter(i => i !== keyword);
      }
      
      hasCache[w] = true;

      function needResearch() {
        if (texts.length !== 10) return false;
        if (w === keyword) return true;
        if (w !== keyword && mod !== 'simple') return true;
      }
    }

    async function batchLoop(turnCount) {
      log(chalk.green('开始第' + turnCount + '轮抓取'));
      const len = result.length;
      if (turnCount === 1) {
        await loopOneWord(keyword);
      } else {
        const arr = result.filter(i => (i.split(' ').length - keyword.split(' ').length <= turnCount && !hasCache[i]));
        if (arr.length) {
          let i = 0;
          while(i < arr.length) {
            const arg = arr[i++];
            if (wordInclude(arg)) {
              log(chalk.yellow((i) + '/' + arr.length), chalk.yellow('第' + turnCount + '轮'));
              await loopOneWord(arg);
            }
          }
        }
      }
      history.push(result.length - len);
      log(chalk.greenBright('第' + turnCount + '轮抓取完成，已抓取' + result.length + '个词'));
    }

    // 如果联想出来的一个词已经没有包含初始词，联想跑偏了，可以结束程序
    function wordInclude(text) {
      const arr = text.split(' ');
      const p = keyword.split(' ');
      return arr.some(i => p.includes(i));
    }

    async function start() {
      log(chalk.green('开始抓取关键词...'));
      let turnCount = 1;
      while (turnCount === 1 || renewRatePass() && turnCount <= maxTurnCount) {
        last = result.length || 1;
        await batchLoop(turnCount++);
      }
      log(chalk.green('关键词已抓取完毕😊'), history);
      await outputFile();
    }

    function formatMind() {
      const cache = {};
      function loop(obj) {
        const name = obj.name;
        cache[name] = true;
        obj.children = (function() {
          if (map[name] && map[name].length && map[name].filter(i => !cache[i]).length) {
            return map[name].filter(i => !cache[i]).map(i => loop({name: i}));
          }
          return [];
        })();
        return obj;
      }
      return loop({name: keyword});
    }

    function renewRatePass() {
      const newLen = result.length - last;
      const rate = newLen/last;
      log(chalk.green('已抓取词' + result.length + '个'), chalk.green('此轮' + newLen + '个'), chalk.green('刷新率: '), chalk.yellow(rate));
      return rate > RATE;
    }

    await start();

    async function outputFile() {
      fs.writeFileSync(keyword + '.txt', parseContent(unique(result)));
      if (needJson) {
        fs.writeFileSync(keyword + '-mind.json', JSON.stringify(formatMind(), null, 2));
      }
      log(chalk.green('文件已生成'));
      log(chalk.green('爬取' + result.length + '个词，共' + history.length + '轮，耗时' + (new Date().getTime() - timestamp)/60000 + 'min'));
    }

    function parseContent(arr) {
      let result = '非长尾词：\n'+ rootWords.join('\n') + '\n\n';
      const LongTailWords = arr.filter(i => !rootWords.includes(i));
      result += '长尾词：\n' + LongTailWords.join('\n') + '\n\n' + '****************************' + '\n\n';
      const two = arr.filter(i => i.split(' ').length <= 2);
      const three = arr.filter(i => i.split(' ').length === 3);
      const more = arr.filter(i => i.split(' ').length > 3);
      result += '两个词以内：\n' + two.join('\n') + '\n\n';
      if (three.length) {
        result += '三个词：\n' + three.join('\n') + '\n\n';
      }
      if (more.length) {
        result += '超过三个词:\n' + more.join('\n');
      }
      return result;
    }

  } catch (error) {
    log(chalk.red(error));
  } finally {
    browser.close();
    process.exit(0);
  }
}

main();